import traceback
import sys
import os
from bs4 import BeautifulSoup
import configparser
import pymysql
from urllib import parse

from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -3)
sPath = BaseDir.get_new_path(TopPath, "download", "zswj",'download', "raw")

gClassName = 'OTHER'
ListSqls = list()
nStart = 1
nCount = 0


# 测试文字输出到文本
def output(value, files='log.txt'):
    f = open(files, 'w', encoding='utf-8')
    f.write(value)
    f.close()


# 数据库链接
def MajorDbConnect():
    cf = configparser.ConfigParser()
    try:
        cf.read(os.path.join(curPath, 'db.ini'))
    except:
        print('读取配置失败:' + traceback.format_exc())
        sys.exit(-1)
    if cf:
        db_host = cf.get('db', 'host')
        db_user = cf.get('db', 'user')
        db_pw = cf.get('db', 'passwd')
        db_name = cf.get('db', 'db')
        db_charset = cf.get('db', 'chartset')
        conn = pymysql.connect(host=db_host, user=db_user, passwd=db_pw, db=db_name, charset=db_charset)
        if conn:
            return conn


# 获取图片名称
def GetFileName(sPath):
    global ListSqls
    print('Start!!!')

    soup = BeautifulSoup(open(sPath, 'r', encoding='utf-8'), "lxml")

    divs = soup.find_all('div', class_='block mode_simple')
    for div in divs:
        img = div.find('img', class_='round_corner')
        src = img.get('src')
        url = 'http://www.libvideo.com' + src

        href = div.find('a').get('href')
        # print(href)
        vid = parse.parse_qs(parse.urlparse(href.replace('[=]', '=')).query)['id'][0]
        sql = "INSERT IGNORE INTO cover(vid, url) VALUES('{vid}', '{url}') ".format(vid=vid, url=url)
        ListSqls.append(sql)

        if len(ListSqls) > 1000:
            InsertIntoDbFromList()


# 插入数据库
def InsertIntoDbFromList():
    global nCount, ListSqls
    dbMsg = None
    conn = MajorDbConnect()
    if conn:
        cur = conn.cursor()
        for sSql in ListSqls:
            try:
                cur.execute(sSql)
                nCount += 1
            except:
                print('* errSql:' + sSql)
                dbMsg = '*errSynax:' + traceback.format_exc()

            if dbMsg:
                output(dbMsg)
                continue

        # print('Succ INSERT DB:%d' %nCount)

        conn.commit()
        conn.close()
        ListSqls = list()


# ----------------------------------------------逻辑部分-----------------------------------------------
# ----------------------------------------------录入数据库wanID----------------------------------------
def main():
    print('path:' + sPath)
    for path, dirNames, fileNames in os.walk(sPath):
        for fileName in fileNames:
            filePath = os.path.join(path, fileName)
            print(filePath)
            GetFileName(filePath)

        # 修改工作存放html的工作目录
        # os.chdir(path)

        # 开始作业

        if len(ListSqls) > 0:
            InsertIntoDbFromList()

        print('DOWN TOTAL:' + str(nCount))


if __name__ == '__main__':
    main()

